In [1]:
%load_ext autoreload
%autoreload 2
In [2]:
import sys
sys.path.append('.')
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt

import pandas as pd

import incense
from incense import ExperimentLoader
from cycler import cycler

Finding experiments

To use incense we first have to instantiate an experiment loader that will enable us to query the database for specific runs.

In [3]:
loader = ExperimentLoader(
    mongo_uri="mongodb://localhost:27017", 
    db_name='sacred'
)
In [4]:
query = {"$and": [
            {"config.autoencoder_type": "normal_dim"},
           
        ]}
experiments=loader.find(query)
In [5]:
experiments
Out[5]:
QuerySet([Experiment(id=9, name=autoencoder_test), Experiment(id=10, name=autoencoder_test), Experiment(id=11, name=autoencoder_test), Experiment(id=12, name=autoencoder_test), Experiment(id=13, name=autoencoder_test), Experiment(id=14, name=autoencoder_test), Experiment(id=15, name=autoencoder_test), Experiment(id=16, name=autoencoder_test), Experiment(id=70, name=autoencoder_test), Experiment(id=71, name=autoencoder_test), Experiment(id=72, name=autoencoder_test), Experiment(id=73, name=autoencoder_test)])
In [6]:
experiments[0].config
Out[6]:
pmap({'batch_size': 256, 'epochs': 250, 'targets_type': 'Mnist', 'seed': 864970147, 'iteration': False, 'autoencoder_type': 'normal_dim'})
In [7]:
experiments.project(on=["config.targets_type","config.targets_type", "config.iteration", "config.autoencoder_type", "config.batch_size"])
Out[7]:
targets_type iteration autoencoder_type batch_size
exp_id
9 Mnist False normal_dim 256
10 Mnist False normal_dim 128
11 Mnist False normal_dim 64
12 Mnist False normal_dim 32
13 10_Targets False normal_dim 256
14 10_Targets False normal_dim 128
15 10_Targets False normal_dim 64
16 10_Targets False normal_dim 32
70 Noisy False normal_dim 256
71 Noisy False normal_dim 128
72 Noisy False normal_dim 64
73 Noisy False normal_dim 32
In [8]:
#experiments=experiments[0:6]
experiments
Out[8]:
QuerySet([Experiment(id=9, name=autoencoder_test), Experiment(id=10, name=autoencoder_test), Experiment(id=11, name=autoencoder_test), Experiment(id=12, name=autoencoder_test), Experiment(id=13, name=autoencoder_test), Experiment(id=14, name=autoencoder_test), Experiment(id=15, name=autoencoder_test), Experiment(id=16, name=autoencoder_test), Experiment(id=70, name=autoencoder_test), Experiment(id=71, name=autoencoder_test), Experiment(id=72, name=autoencoder_test), Experiment(id=73, name=autoencoder_test)])
In [9]:
def print_imm(imgs,name):
    n = len(imgs[0]) # how many digits we will display
    if name:
        plt.figure(figsize=(2,0.5))
        plt.text(0.1, 0.1, name, fontsize=12) 
        plt.show()
    plt.figure(figsize=(2*len(imgs[0]), 2 * len(imgs) + 2))
    for i in range(n):
        for j in range(len(imgs)):
            # display original
            ax = plt.subplot(len(imgs), n, i + 1 + j * n)
            
            plt.imshow(imgs[j][i].reshape(28, 28))
            
            plt.gray()
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

    plt.show()
In [10]:
def compare(data):
    evaluations_in_classifier=[]
    evaluations_feature_classifier=[]
    evaluations_out_classifier=[]
    autoencoder=[]
    predictions=[]
    pickle_artifact=[]
    
    for exp in experiments:
        #print(exp.id,exp.config)
        pickle_artifact = exp.artifacts[data].as_type(incense.artifact.PickleArtifact)
        predictions=pd.read_pickle(pickle_artifact.file,compression='gzip')
        
        evaluations_in_classifier.append(predictions['evaluations_in_classifier'])
        evaluations_feature_classifier.append(predictions['evaluations_feature_classifier'])
        evaluations_out_classifier.append(predictions['evaluations_out_classifier'])
        autoencoder.append(predictions['evaluations_autoencoder'])
        

    print(data)
    names=[]
    for exp in experiments:
        names.append(exp.id)
    
    plt.rc('axes', prop_cycle=(cycler('color', ['r', 'g', 'b', 'y','r', 'g', 'b', 'y','r', 'g', 'b', 'y','r', 'g', 'b', 'y']) +
                           cycler('linestyle', ['-','-','-','-', '--','--','--','--', ':',':',':',':' ,'-.','-.','-.','-.'])))
    
    plt.figure(figsize=(30,15))    

    plt.subplot(2, 3,1)
    plt.title("Accuracy over iterations evaluations_in_classifier") 
    i=0
    for exp in evaluations_in_classifier:
        plt.plot([item[1] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
    plt.subplot(2, 3,2)
    plt.title("Accuracy over iterations evaluations_feature_classifier") 
    i=0
    for exp in evaluations_feature_classifier:
        plt.plot([item[1] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
    
    plt.subplot(2, 3,3)
    plt.title("Accuracy over iterations evaluations_out_classifier") 
    i=0
    for exp in evaluations_out_classifier:
        plt.plot([item[1] for item in exp], label=names[i])
        i=i+1
    plt.legend()  

    plt.subplot(2, 3,4)
    plt.title("Loss over iterations autoencoder") 
    i=0
    for exp in autoencoder:
        plt.plot([item[0] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
  
    
    
    plt.subplot(2, 3,6)
    plt.title("mae iterations autoencoder") 
    i=0
    for exp in autoencoder:
        plt.plot([item[2] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
    plt.show()
    
    
In [11]:
name_list=['predictions_df_0','predictions_df_10','predictions_df_20','predictions_df_30','predictions_df_40','predictions_df_50','predictions_df_60','predictions_df_70','predictions_df_80','predictions_df_90','predictions_df_100']
#name_list=['predictions_df_0','predictions_df_10','predictions_df_20']
In [12]:
for index in name_list:
    compare(index)
predictions_df_0
predictions_df_10
predictions_df_20
predictions_df_30
predictions_df_40
predictions_df_50
predictions_df_60
predictions_df_70
predictions_df_80
predictions_df_90
predictions_df_100
In [13]:
for data in name_list:
    df=[]
    plt.figure(figsize=(2,0.5))
    plt.text(0.1, 0.1, data, fontsize=12) 
    plt.show()
    num=['7','2','1','0']
    df = pd.concat([pd.read_pickle(exp.artifacts[data].as_type(incense.artifact.PickleArtifact).file,compression='gzip')['predictions'] for exp in experiments],axis=1,keys=[str(exp.id) for exp in experiments])
    for i in range(2):
        print_imm(np.array(df.applymap(lambda x: x[i])),num[i])
    df=[]